﻿// ParseLib - An open source library for natural language processing.
// Copyright (C) Kais Dukes, 2012. E-mail: kais@kaisdukes.com
// http://parselib.org

using System.Collections.Generic;
using System.IO;
using System.Linq;
using Ionic.Zip;
using ParseLib.Tagging;

namespace ParseLib.Corpora.Brown
{
    public class BrownCorpus : Corpus
    {
        private TextualUnit[] _sentences;

        public override TextualUnit this[int index]
        {
            get { return _sentences[index]; }
        }

        public override void Load()
        {
            using (var zipFile = new ZipFile(Source))
            {
                Load(zipFile);
            }
        }

        private void Load(IEnumerable<ZipEntry> zipFile)
        {
            var entries = zipFile
                .Where(x => Path.GetFileName(x.FileName).Length == 4)
                .OrderBy(x => x.FileName)
                .ToList();

            PartOfSpeechTagset = new PartOfSpeechTagset();
            var brownReader = new BrownReader(this);
            var sentences = new List<TextualUnit>();

            foreach (var entry in entries)
            {
                using (var reader = new StreamReader(entry.OpenReader()))
                {
                    string line;
                    while ((line = reader.ReadLine()) != null)
                    {
                        var sentence = brownReader.ReadSentence(line);
                        if (sentence != null)
                        {
                            sentences.Add(sentence);
                        }
                    }
                }
            }

            _sentences = sentences.ToArray();
            TextualUnitCount = _sentences.Length;
        }
    }
}